# -*- encoding:utf-8 -*-
import os
import time
import datetime

import requests
import xlwt
import xlrd
from xlutils.copy import copy
from constant import constant_define

from dao.MySqlDAL import MySqlDAL
import utils.CookieUtils as cu

__author__ = 'shudong.msd'

'''
    抓取各类热榜

    输出关注排行榜 新增
    http://xueqiu.com/stock/screener/screen.json?category=SH&orderby=follow7d&order=desc&page=1&follow7d=ALL&_=1443108548354
    http://xueqiu.com/stock/screener/screen.json?category=SH&size=3000&order=desc&orderby=follow7d&current=ALL&pct=ALL&follow7d=ALL&_=1443142974231

    关注排行榜 最热
    http://xueqiu.com/stock/screener/screen.json?category=SH&orderby=follow&order=desc&page=1&follow=ALL&_=1443143224743
    http://xueqiu.com/stock/screener/screen.json?category=SH&size=10&order=desc&orderby=follow&current=ALL&pct=ALL&follow=ALL&_=1443143315597

    讨论排行榜 新增
    http://xueqiu.com/stock/screener/screen.json?category=SH&orderby=tweet7d&order=desc&page=1&tweet7d=ALL&_=1443143458886
    http://xueqiu.com/stock/screener/screen.json?category=SH&size=10&order=desc&orderby=tweet7d&current=ALL&pct=ALL&tweet7d=ALL&_=1443143343427

    讨论排行榜 最热
    http://xueqiu.com/stock/screener/screen.json?category=SH&orderby=tweet&order=desc&page=1&tweet=ALL&_=1443143505664
    http://xueqiu.com/stock/screener/screen.json?category=SH&size=10&order=desc&orderby=tweet&current=ALL&pct=ALL&tweet=ALL&_=1443143365811

    分享交易排行榜_本周新增
    http://xueqiu.com/stock/screener/screen.json?category=SH&orderby=deal7d&order=desc&page=1&deal7d=ALL&_=1443143537669
    http://xueqiu.com/stock/screener/screen.json?category=SH&size=10&order=desc&orderby=deal7d&current=ALL&pct=ALL&deal7d=ALL&_=1443143393179

    分享交易排行榜_最热门
    http://xueqiu.com/stock/screener/screen.json?category=SH&orderby=deal&order=desc&page=1&deal=ALL&_=1443143569881
    http://xueqiu.com/stock/screener/screen.json?category=SH&size=10&order=desc&orderby=deal&current=ALL&pct=ALL&deal=ALL&_=1443143412340


'''


class ShareRankCrawler(object):

    # 新增排行榜可全部读取不出错
    __recordCount = 3000

    # 热度排行榜最多读取到1191条，后期可能会好转
    __hotRecordCount = 1135

    __hotRecordCount_asc = 1000

    # tablename
    __tablename = 'tb_share_rank'

    __code_list = ['GB2312','GBK','utf-8','ascii','ANSI']

    __headers = {
            'Accept':'application/json, text/javascript, */*; q=0.01',
            'Accept-Encoding':'gzip, deflate, sdch',
            'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
            'cache-control':'no-cache',
            'Connection':'keep-alive',
            'User-Agent': constant_define.user_agent,
            'Referer':'http://xueqiu.com/hq/screener/CN',
            'Host':'xueqiu.com',
            'X-Requested-With':'XMLHttpRequest',
            'Cookie': cu.init_cookie()
    }



    def set_target(self,target):
        self.__target = target

    # 得到响应信息
    def __get_response(self,type=0,isHotAsc=False):
        if type in (0,2,4):
            return requests.get(self.__target,params={'size':self.__recordCount},headers=self.__headers).json()
        else:
            if isHotAsc:
                return requests.get(self.__target,params={'size':self.__hotRecordCount_asc},headers=self.__headers).json()
            else:
                return requests.get(self.__target,params={'size':self.__hotRecordCount},headers=self.__headers).json()

    # 获得所有的数据dict

    def get_final_list(self,type=0,isHotAsc=False):
        res_dic = self.__get_response(type,isHotAsc)
        # self.__recordCount = res_dic['count']
        res_list = res_dic['list']

        tmp_fin_list = list()
        if type ==0 :
            for row in res_list:
                tmp_list = list()
                tmp_list.append(row['name'])
                tmp_list.append(row['symbol'])
                tmp_list.append(row['follow7d'])
                if tmp_list in tmp_fin_list:
                    continue
                tmp_fin_list.append(tmp_list)
        elif type == 1:
            for row in res_list:
                tmp_list = list()
                tmp_list.append(row['name'])
                tmp_list.append(row['symbol'])
                tmp_list.append(row['follow'])
                if tmp_list in tmp_fin_list:
                    continue
                tmp_fin_list.append(tmp_list)
        elif type == 2:
            for row in res_list:
                tmp_list = list()
                tmp_list.append(row['name'])
                tmp_list.append(row['symbol'])
                tmp_list.append(row['tweet7d'])
                if tmp_list in tmp_fin_list:
                    continue
                tmp_fin_list.append(tmp_list)
        elif type == 3:
            for row in res_list:
                tmp_list = list()
                tmp_list.append(row['name'])
                tmp_list.append(row['symbol'])
                tmp_list.append(row['tweet'])
                if tmp_list in tmp_fin_list:
                    continue
                tmp_fin_list.append(tmp_list)
        elif type == 4:
            for row in res_list:
                tmp_list = list()
                tmp_list.append(row['name'])
                tmp_list.append(row['symbol'])
                tmp_list.append(row['deal7d'])
                if tmp_list in tmp_fin_list:
                    continue
                tmp_fin_list.append(tmp_list)
        elif type == 5:
            for row in res_list:
                tmp_list = list()
                tmp_list.append(row['name'])
                tmp_list.append(row['symbol'])
                tmp_list.append(row['deal'])
                if tmp_list in tmp_fin_list:
                    continue
                tmp_fin_list.append(tmp_list)

        # 排序  按照字典的 value 排序 返回列表
        zfinal_list = sorted(tmp_fin_list, key=lambda d: d[2],reverse=True)
        #print zfinal_list[0]
        # [(,),(,)]
        return zfinal_list


    def __getTitle(self,type=0):
        if type ==0 :
            return [u'股票',u'代码',u'一周新增关注']
        elif type == 1:
            return [u'股票',u'代码',u'累计关注人数']
        elif type == 2:
            return [u'股票',u'代码',u'一周新增讨论数']
        elif type == 3:
            return [u'股票',u'代码',u'累计讨论次数']
        elif type == 4:
            return [u'股票',u'代码',u'新增分享交易数量']
        elif type == 5:
            return [u'股票',u'代码',u'最热分享交易数量']



    # 获得完整信息列表(包括表头和表体)
    def __get_table(self,type=0):
        if self.__target == "":
            print u"请先输入目标网址"
            return
        # table的存储形式是:[{"表头1":"对应数据","表头2":"对应数据"},{},...]
        table = []
        titleList = self.__getTitle(type)
        bodyList = self.get_final_list(type)

        for row in bodyList:
            recordic = {}
            row[1] = '$'+row[0]+'('+row[1]+')$'
            for i in xrange(0,len(row)):
                recordic[titleList[i]] = row[i]
            table.append(recordic)
        return table

    # 输出到Excel
    def writeToExcel(self,filePath,type=0,pos=0):
        workbook = ""
        worksheet = ""
        if os.path.isfile(filePath):
            workbook = copy(xlrd.open_workbook(filePath))
            worksheet = workbook.get_sheet(0)
        else:
            workbook = xlwt.Workbook(encoding = 'utf-8')
            worksheet = workbook.add_sheet('sheet1')

        table = self.__get_table(type)
        colnames =  self.__getTitle(type)

        #
        # 用到python 操作Excel表格，使用xlwt插件，发现里面write_merge（）函数用着最灵活
        # write_merge(x, x + h, y, w + y, string, sytle)
        # x表示行，y表示列，w表示跨列个数，h表示跨行个数，string表示要写入的单元格内容，style表示单元格样式。
        # 注意，x，y，w，h，都是以0开始计算的。
        #
        # 网上中文资料不多还很多转载同样的，我是研究源码知道的。
        # 插入图像insert_bitmap()函数可以插入图像，
        # insert_bitmap(img, x, y, x1, y1, scale_x=0.8, scale_y=1)
        # img表示要插入的图像地址，
        # x表示行
        # y表示列
        # x1表示相对原来位置向下偏移的像素
        # y1表示相对原来位置向右偏移的像素
        # scale_x表示相对原图宽的比例
        # scale_y表示相对原图高的比例

        # 合并单元格
        if type == 0:
            worksheet.write_merge(0,0,pos*5,pos*5+3,u'关注排行榜_最新')
        elif type == 1:
            worksheet.write_merge(0,0,pos*5,pos*5+3,u'关注排行榜_最热')
        elif type == 2:
            worksheet.write_merge(0,0,pos*5,pos*5+3,u'讨论排行榜_最新')
        elif type == 3:
            worksheet.write_merge(0,0,pos*5,pos*5+3,u'讨论排行榜_最热')
        elif type == 4:
            worksheet.write_merge(0,0,pos*5,pos*5+3,u'分享交易排行榜_最新')
        elif type == 5:
            worksheet.write_merge(0,0,pos*5,pos*5+3,u'分享交易排行榜_最热')

        #输出表头
        i = 0
        for item in colnames:
            worksheet.write(1, pos*5+i,item)
            i += 1

        #输出内容
        index = 2
        for row in table:
            j = 0
            for col in colnames:
                worksheet.write(index, pos*5+j,row[col])
                j += 1
            index += 1

        workbook.save(filePath)

    # 写入db
    def writeToDB(self,type=0,isHotAsc=False):
        sqlUtil = MySqlDAL()
        # [(,),(,)]
        stock_list = self.get_final_list(type,isHotAsc)

        data_keys = ['stock_id','num','type','date','update_time']

        # [u'股票',u'代码',u'一周新增关注']
        data_source = list()
        for row in stock_list:
            tmp_list = (row[1],row[2],type,datetime.date.today(),time.time())
            data_source.append(tmp_list)

        return sqlUtil.insert_many_batch(data_keys,data_source,self.__tablename,50)

    # 更新db
    def updateDB(self,type=0):
        sqlUtil = MySqlDAL()
        # [(,),(,)]
        stock_list = self.get_final_list(type)

        # data_source=[(修改值1),(2),(3)...(过滤条件1),(2)...] ,

        data_keys = ['num','update_time']
        filter_collection_key=['stock_id','date','type']

        # row[u'股票',u'代码',u'一周新增关注']
        data_source = list()
        for row in stock_list:
            tmp_list = (row[2],time.time(),row[1],datetime.date.today(),type)
            data_source.append(tmp_list)

        return sqlUtil.update_many_batch(data_keys,data_source,self.__tablename,filter_collection_key,50)

